package com.yaoandw.givebook;

import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.Constructor;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;

import com.yaoandw.crawler.PageIndexTool;
import com.yaoandw.givebook.site.BxwxorgImpl;
import com.yaoandw.givebook.site.ISite;
import com.yaoandw.givebook.site.Txdzscom;
import com.yaoandw.storage.Frontier;
import com.yaoandw.storage.FrontierFactory;
import com.yaoandw.storage.SimpleCrawlUrl;
import com.yaoandw.util.ClassUtils;

public class BookCrawler implements Crawler{
//	static{
//		initLog4j();
//	}
	private static Logger logger = Logger.getLogger(BookCrawler.class);
//public static String baseUrl = "http://www.bxwx.org";
	public static String baseUrl = "http://www.bxwx.org/binfo/8/8823.htm";
	
	public static String[] baseUrls = new String[]{BxwxorgImpl.SITE_URL,Txdzscom.SITE_URL};
	
	public static Properties configuration = new Properties();
	public static String configCrawlMode = "text";
	public static String configDownload = "false";
	public static String configFileSuffix = "txt";
	public static int configCrawlLayer = 5;
	public static int poolSize = 200;
	public static Map<String,Class> siteImpl = new HashMap<String,Class>();
	
	public BookCrawler(){
		
	}
	
	BlockingQueue<Runnable> blockingQueue = new LinkedBlockingQueue<Runnable>();
	ThreadPoolExecutor threadPool;
	
	DispatcherThread dispatcher;

	private void initConfiguration(){
		InputStream is = this.getClass().getResourceAsStream("crawl_config.properties");
		try {
			configuration.load(is);
			configCrawlMode = configuration.getProperty("crawl_mode");
			configDownload = configuration.getProperty("download");
			configFileSuffix = configuration.getProperty("file_suffix");
			configCrawlLayer = Integer.valueOf(configuration.getProperty("crawl_layer"));
		} catch (IOException e) {
			logger.error("",e);
		}
		logger.debug(configuration.toString());
	}
	/**
	 * 抓取过程
	 * 
	 * @return
	 * @param seeds
	 */
	public void run() {
		Constants.status = Constants.status_running;
		initConfiguration();
		if(!initImplClass())
			return;
		boolean b = PageIndexTool.beginIndex();
		Frontier frontier = FrontierFactory.getFrontier(Constants.bdbDir);
		if(frontier == null){
			return;
		} 
		threadPool = new ThreadPoolExecutor(poolSize, poolSize, 3, TimeUnit.SECONDS, blockingQueue,new ThreadPoolExecutor.CallerRunsPolicy()){
			@Override
			protected void afterExecute(Runnable r, Throwable t) {
				logger.debug(Thread.currentThread().getId()+" executed");
				super.afterExecute(r, t);
			}
		};
		dispatcher = new DispatcherThread(this);
		new Thread(dispatcher).start();
		for(String visitUrl : baseUrls){
			logger.debug("start visitUrl:" + visitUrl);
			if (visitUrl == null)
				continue;
			Class cl = siteImpl.get(visitUrl);
			if(cl == null){
				logger.debug("cl="+cl+",continue.");
				continue;
			}
			try {
				Constructor ctor[]=cl.getDeclaredConstructors();
				Class cx[]=ctor[0].getParameterTypes(); 
				threadPool.execute((Runnable)cl.getConstructor(cx).newInstance(new Object[]{new SimpleCrawlUrl(visitUrl,0)}));
//				ISite site = (ISite)cl.getMethod("getInstance").invoke(null);
//				site.index();
			} catch (Exception e) {
				logger.error("",e);
			} 
		}
		
		while(true){
			logger.debug("getActiveCount:"+threadPool.getActiveCount()+",url cnt:"+BxwxorgImpl.getCrawledCnt()+",blockingQueue size:"+blockingQueue.size());
			if(threadPool.getActiveCount() <= 0 && Constants.status_stoped.equals(Constants.status)){
				threadPool.shutdown();
				break;
			}
			try {
//				synchronized (this) {
//					wait();
//				}
				Thread.sleep(5000);
			} catch (InterruptedException e) {
				logger.error("",e);
			}
		}
		//
		if(threadPool.isShutdown()){
			logger.debug("closeIndex............");
			PageIndexTool.closeIndex();
			frontier.close();
			logger.debug(BxwxorgImpl.getCrawledCnt());
		}
	}

	public void assign(Class cl,SimpleCrawlUrl nextUrl) {
		try {
			Constructor ctor[]=cl.getDeclaredConstructors();
			Class cx[]=ctor[0].getParameterTypes(); 
			threadPool.execute((Runnable)cl.getConstructor(cx).newInstance(new Object[]{nextUrl}));
		} catch (Exception e) {
			logger.error("",e);
		}
	}
	public void assign(SimpleCrawlUrl nextUrl) {
		Class cl = getUrlClass(nextUrl.getOriUrl());
		if(cl != null)
			assign(cl,nextUrl);
	}
	private Class getUrlClass(String oriUrl) {
		for(String url : baseUrls){
			if(oriUrl.startsWith(url))
				return siteImpl.get(url);
		}
		return null;
	}
	public int getActivePoolCount(){
		return threadPool.getActiveCount();
	}

	public int getMaxPoolSize() {
		return threadPool.getMaximumPoolSize();
	}
	public boolean poolIsShutdown() {
		return threadPool.isShutdown();
	}
	private boolean initImplClass() {
		for(Class cl : ClassUtils.getAllClassByInterface(ISite.class))
			try {
				siteImpl.put(cl.getField("SITE_URL").get(null).toString(), cl);
			} catch (IllegalArgumentException e) {
				logger.error("",e);
				return false;
			} catch (SecurityException e) {
				logger.error("",e);
				return false;
			} catch (IllegalAccessException e) {
				logger.error("",e);
				return false;
			} catch (NoSuchFieldException e) {
				logger.error("",e);
				return false;
			}
		return true;
	}
	public void pauseCarawler() {
		Constants.status = Constants.status_paused;
	}
	public void continueCarawler() {
		Constants.status = Constants.status_running;
		synchronized (dispatcher) {
			dispatcher.notify();
		}
	}
	public void stopCarawler() {
		Constants.status = Constants.status_stoped;
	}

	public static void main(String[] args){
		BookCrawler crawler = new BookCrawler();
		new Thread(crawler).start();
	}
}
